# 爬取京东数据
# 获取商品信息

import requests
from bs4 import BeautifulSoup
from openpyxl import Workbook


def get_commodity(keyword, page):
    ret_list = []
    start_url = 'https://search.jd.com/Search?keyword={}&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&page={}&click=0' \
        .format(str(keyword), str(page))
    headers = {
        # 标记了请求从什么设备，什么浏览器上发出
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0',
        'accept': 'text/html',
        'Access - Control - Allow - Origin': 'https: //search.jd.com',
        'Connection': 'keep - alive',
        'Content - Encoding': 'gzip',
        'Content - Type': 'text / html;charset = utf - 8',
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2'
    }
    response = requests.request('GET', start_url, headers=headers)
    response.encoding = response.apparent_encoding
    soup = BeautifulSoup(response.text, "lxml")
    list_html = soup.findAll("li", {"class", "gl-item"})
    if len(list_html) >= 0:
        for html in list_html:
            goods = {}
            # 商品详情url
            detail_url = html.find("div", {"class", "p-img"}).find("a").get("href")
            # 商品标题
            title = html.find("div", {"class", "p-img"}).find("a").get("title")
            # 商品图片
            goods_image = html.find("div", {"class", "p-img"}).find("img").get("data-lazy-img")
            # 商品规格
            goods_spec = html.find("div", {"class", "p-scroll"}).find("a").get("title")
            # 商品价格
            goods_price = html.find("div", {"class", "p-price"}).find("i").text
            # spkCode
            sku_code = html.find("div", {"class", "ps-wrap"}).find("img").attrs['data-sku']
            # spuName
            sku_name_em = html.find("div", {"class", "p-name p-name-type-2"}).find("a").findAll('em')
            sku_name_font = html.find("div", {"class", "p-name p-name-type-2"}).find("a").findAll('font')
            spu_name = ''
            if len(sku_name_em) > 0:
                for em in sku_name_em:
                    spu_name += em.text
            if len(sku_name_font) > 0:
                for em in sku_name_font:
                    spu_name += em.text
            # 商店
            shop_name = html.find("div", {"class", "p-shop"}).find("span").text
            # 店铺url
            shop_url = html.find("div", {"class", "p-shop"}).find("a").attrs['href']
            # 评论人数
            commit = html.find("div", {"class", "p-commit"}).find('strong').find("a").text

            goods.setdefault('A', sku_code)
            goods.setdefault('B', goods_image)
            goods.setdefault('C', goods_spec)
            goods.setdefault('D', goods_price)
            goods.setdefault('E', spu_name)
            goods.setdefault('F', detail_url)
            goods.setdefault('G', title)
            goods.setdefault('H', shop_name)
            goods.setdefault('I', shop_url)
            goods.setdefault('G', 0 if commit else commit)
            ret_list.append(goods)
    return ret_list


# 保存到excel
def import_excel(file_name, data_list):
    if not (data_list and len(data_list) > 0):
        return
    # 内存中创建一个workBook
    wb = Workbook()
    # 获取当前活跃的worksheet,默认取第一个
    ws = wb.active
    # 设置Excel头
    ws.title = file_name
    # 设置第一行的标题
    ws.column_dimensions.width = 120
    ws.cell(row=1, column=1).value = '商品id'
    ws.cell(row=1, column=2).value = '商品图片'
    ws.cell(row=1, column=3).value = '商品规格'
    ws.cell(row=1, column=4).value = '商品价格'
    ws.cell(row=1, column=5).value = '商品名称'
    ws.cell(row=1, column=6).value = '商品详情链接'
    ws.cell(row=1, column=7).value = '商品标题'
    ws.cell(row=1, column=8).value = '店铺名称'
    ws.cell(row=1, column=9).value = '店铺url'
    ws.cell(row=1, column=10).value = '评论人数'
    for data in data_list:
        ws.append(data)
    # 保存目录
    wb.save(file_name + ".xlsx")
    wb.close()


print(get_commodity("华为Mate40", 1))
# import_excel('华为Mate40商品信息', get_commodity("华为Mate40", 1))
